# coding:gb2312
import re
import time, requests
from datetime import datetime
from multiprocessing import Pool
from tianyancha.mongo import mongoDb
from pyquery import PyQuery as pq

ipUrl = "http://www.kuaidaili.com/proxylist/1/"


def getIpss(ipUrl):
    jq = pq(ipUrl, headers=header)
    ips = [{
        "ip": jq(tr).find('td:nth-child(1)').text(),
        "port": jq(tr).find('td:nth-child(2)').text(),
        "http": jq(tr).find('td:nth-child(3)').text(),
        "method": jq(tr).find('td:nth-child(4)').text(),
        "location": jq(tr).find('td:nth-child(5)').text(),
        "speed": jq(tr).find('td:nth-child(6)').text(),
        "lastTime": jq(tr).find('td:nth-child(7)').text()
    } for tr in
        jq('tbody tr')]
    try:
        with mongoDb('ipSpiders') as iptable:
            iptable.insert(ips)
        print('this page:{0} is sipdering ok.........................'.format(ipUrl))
    except:
        print('this page:{0} is sipdering fail.......................'.format(ipUrl))
        pass


def validIp(ip):
    # "http":?"http://10.10.1.10:3128"
    proxies = {"http": "http://%(ip)s:%(port)s" % ip}
    print('handlering :{0}'.format(proxies))
    # <editor-fold desc="Description">
    startime = datetime.now()
    # </editor-fold>
    try:
        text = requests.get('http://1212.ip138.com/ic.asp', proxies=proxies, timeout=30).text
        try:
            ip = re.match(r'.*\[(.*?)\].*', text, re.S).group(1)
            validTime = datetime.now() - startime;
            ip["validTime"] = validTime
            try:
                with mongoDb('ipAgents') as agent:
                    agent.insert_one(ip)
                print('this proxy:%s is valid ok and ip :{%s} usetime :%s seconds...........................' % (
                    proxies, ip, str(validTime)))
            except:
                pass
        except:
            print('this proxy:{0} is valid fail.........................'.format(proxies))
    except:
        print('this proxy:{0} is valid timeout.........................'.format(proxies))
    pass


def xcgetIp(page):
    print('this is xcgetIp(%s)...........' % page)
    jq = pq(page, headers=header)
    for tr in jq('#ip_list tr')[1:]:
        ip = {
            "ip": jq(tr).find('td:nth-child(2)').text(),
            "port": jq(tr).find('td:nth-child(3)').text(),
            "locate": jq(tr).find('td:nth-child(4)').text(),
            "method": jq(tr).find('td:nth-child(6)').text().lower()
        }
        validIp(ip)


def kdgetIp(page):
    print('this is xcgetIp(%s)...........' % page)
    header = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Encoding": "gzip, deflate, sdch",
        "Accept-Language": "zh-CN,zh;q=0.8",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Cookie": "channelid=0; sid=1491790526603378; _ydclearance=e1b6ed226414138467323bd1-765c-402f-a8d5-3e9aa5864fff-1491816619; _ga=GA1.2.1161540.1491790588; Hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1491790588; Hm_lpvt_7ed65b1cc4b810e9fd37959c9bb51b31=1491809421",
        "Host": "www.kuaidaili.com",
        "Pragma": "no-cache",
        "Referer": "http://www.kuaidaili.com/",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.2561.400 QQBrowser/9.6.10822.400",
    }
    jq = pq(page, headers=header)
    ips = [{
        "ip": jq(tr).find('td:nth-child(1)').text(),
        "port": jq(tr).find('td:nth-child(2)').text(),
        "http": jq(tr).find('td:nth-child(3)').text(),
        "method": jq(tr).find('td:nth-child(4)').text(),
        "location": jq(tr).find('td:nth-child(5)').text(),
        "speed": jq(tr).find('td:nth-child(6)').text(),
        "lastTime": jq(tr).find('td:nth-child(7)').text()
    } for tr in jq('tbody tr')]
    with mongoDb('ipSpiders') as ipTable:
        ipTable.insert(ips)


if __name__ == '__main__':
    # pages = ["http://www.kuaidaili.com/proxylist/%s/" % str(i) for i in range(1, 11)]
    # p = Pool(4)
    # p.map(kdgetIp, pages)
    # p.close()
    # p.join()
    p = Pool(8)
    with mongoDb('ipSpiders') as ipTable:
        p.map(validIp, ipTable.find())
    p.close()
    p.join()
